Machine learning work steps through python¶

Import all useable library¶

# import relevant modules
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.formula.api as sn
import scipy.stats as stats
from matplotlib.backends.backend_pdf import PdfPages
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from statsmodels.stats.outliers_influence import variance_inflation_factor
from patsy import dmatrices
%matplotlib inline

# Ignore warnings
import warnings
warnings.filterwarnings('ignore')

# Settings
pd.set_option('display.max_columns', None)
np.set_printoptions(threshold=np.nan)
np.set_printoptions(precision=3)
sns.set(style="darkgrid")
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

Load the Data set.¶

custdata_df = pd.read_excel("Data Set.xlsx", sheetname="customer_dbase")

custdata_df.sample(5)

# Find column information in the dataframe.
custdata_df.columns

Index(['custid', 'region', 'townsize', 'gender', 'age', 'agecat', 'birthmonth',
       'ed', 'edcat', 'jobcat',
       ...
       'owncd', 'ownpda', 'ownpc', 'ownipod', 'owngame', 'ownfax', 'news',
       'response_01', 'response_02', 'response_03'],
      dtype='object', length=130)

Creating Dependent Y column¶

#To create Y we need to sumup cardspent(first card spent amount) and card2spent(Second card spent amount)
custdata_df['totalspend'] = custdata_df['cardspent'] + custdata_df['card2spent']

custdata_df.head()

# Now Run pandas profiling to see the data audit reports

import pandas_profiling
pandas_profiling.ProfileReport(custdata_df)

Drop Variables¶

#Drop cardspent(first card spent amount) and card2spent(Second card spent amount) because its not adding any value
#Also  we are droping "custid","birthmonth" bacuse cust id is unique and birth month is not adding any value

custdata_df.drop(["cardspent","card2spent","custid"],axis=1, inplace=True)

#Item count need to drop promary and secondary Iem count

Check linearlity of data:¶

Use scatter plot by using 'df.plot' or if you want a linear line you can use 'sns.lmplot' through seaborn.

# Seaborn scatter plot with regression line
# aspect=1.5, scatter_kws={'alpha':0.2}) - You can choose this option too.
sns.lmplot(x='income', y='totalspend', data=custdata_df)

<seaborn.axisgrid.FacetGrid at 0xba09c88>

So we can see above most data points are near to line which means our data is normal and we can go ahed for further process.

# Creating Categorical List= Contains categorical variables...

for x in ['region','townsize','gender','agecat','edcat','birthmonth','jobcat','union','employ','empcat','retire',
          'inccat','default','jobsat','marital','spousedcat','homeown','hometype','address','addresscat','cars','carown',
          'cartype','carcatvalue','carbought','carbuy','commute','commutecat','commutecar',
          'commutemotorcycle','commutecarpool','commutebus','commuterail','commutepublic','commutebike','commutewalk',
         'commutenonmotor','telecommute','reason','polview','polparty','polcontrib','vote','card','cardtype','cardbenefit',
         'cardfee','cardtenure','cardtenurecat','card2','card2type','card2benefit','card2fee','card2tenure','card2tenurecat',
         'active','bfast','churn','tollfree','equip','callcard','wireless','multline','voice','pager','internet','callid',
         'callwait','forward','confer','ebill','owntv','ownvcr','owndvd','owncd','ownpda','ownpc','ownipod','owngame','ownfax',
        'news','response_01','response_02','response_03']:
    custdata_df[x]=custdata_df[x].astype('object')

Separate numerical and categorical variable¶

To do that you need to apply for loop along with if condition.

# Find numerical variable in Data frame.
# This will return a list
numeric_var_names = [key for key in dict(custdata_df.dtypes) if dict(custdata_df.dtypes)[key] in ['float64', 'int64', 'float32', 'int32']]

# Find Categorical variable in Data frame
cat_var_names = [key for key in dict(custdata_df.dtypes) if dict(custdata_df.dtypes)[key] in ['object']]

#Print the data frame
print( numeric_var_names)
print(cat_var_names)

['age', 'ed', 'income', 'lninc', 'debtinc', 'creddebt', 'lncreddebt', 'othdebt', 'lnothdebt', 'spoused', 'reside', 'pets', 'pets_cats', 'pets_dogs', 'pets_birds', 'pets_reptiles', 'pets_small', 'pets_saltfish', 'pets_freshfish', 'carvalue', 'commutetime', 'tenure', 'longmon', 'lnlongmon', 'longten', 'lnlongten', 'tollmon', 'lntollmon', 'tollten', 'lntollten', 'equipmon', 'lnequipmon', 'equipten', 'lnequipten', 'cardmon', 'lncardmon', 'cardten', 'lncardten', 'wiremon', 'lnwiremon', 'wireten', 'lnwireten', 'hourstv', 'totalspend']
['region', 'townsize', 'gender', 'agecat', 'birthmonth', 'edcat', 'jobcat', 'union', 'employ', 'empcat', 'retire', 'inccat', 'default', 'jobsat', 'marital', 'spousedcat', 'homeown', 'hometype', 'address', 'addresscat', 'cars', 'carown', 'cartype', 'carcatvalue', 'carbought', 'carbuy', 'commute', 'commutecat', 'commutecar', 'commutemotorcycle', 'commutecarpool', 'commutebus', 'commuterail', 'commutepublic', 'commutebike', 'commutewalk', 'commutenonmotor', 'telecommute', 'reason', 'polview', 'polparty', 'polcontrib', 'vote', 'card', 'cardtype', 'cardbenefit', 'cardfee', 'cardtenure', 'cardtenurecat', 'card2', 'card2type', 'card2benefit', 'card2fee', 'card2tenure', 'card2tenurecat', 'active', 'bfast', 'churn', 'tollfree', 'equip', 'callcard', 'wireless', 'multline', 'voice', 'pager', 'internet', 'callid', 'callwait', 'forward', 'confer', 'ebill', 'owntv', 'ownvcr', 'owndvd', 'owncd', 'ownpda', 'ownpc', 'ownipod', 'owngame', 'ownfax', 'news', 'response_01', 'response_02', 'response_03']

#Convert a list in to dataframe 
#Information about numericvariable in custdata_df

custdata_df_num=custdata_df[numeric_var_names]
custdata_df_num.head(5)

#Convert a list in to dataframe 
#Information about categorical variable in custdata_df

custdata_df_cat=custdata_df[cat_var_names]
custdata_df_cat.head(5)

Creating Data audit Report¶

# Use a general function that returns multiple values
def var_summary(x):
    return pd.Series([x.count(), x.isnull().sum(), x.sum(), x.mean(), x.median(),  x.std(), x.var(), x.min(), x.dropna().quantile(0.01), x.dropna().quantile(0.05),x.dropna().quantile(0.10),x.dropna().quantile(0.25),x.dropna().quantile(0.50),x.dropna().quantile(0.75), x.dropna().quantile(0.90),x.dropna().quantile(0.95), x.dropna().quantile(0.99),x.max()], 
                  index=['N', 'NMISS', 'SUM', 'MEAN','MEDIAN', 'STD', 'VAR', 'MIN', 'P1' , 'P5' ,'P10' ,'P25' ,'P50' ,'P75' ,'P90' ,'P95' ,'P99' ,'MAX'])

num_summary=custdata_df_num.apply(lambda x: var_summary(x)).T

num_summary

def cat_summary(x):
    return pd.Series([x.count(),x.isnull().sum(),x.value_counts(),x.unique()],
                    index=['N','NMISS','ColumnNames','UniqueValues'])

cat_summary=custdata_df_cat.apply(lambda x:cat_summary(x)).T

cat_summary

Handling Outliers¶

There are some extreame high or extream low value which need to rectify so that it will not have impact on our model¶

#Handling Outliers for numerical data - Through function

def outlier_capping(x):
    x = x.clip_upper(x.quantile(0.99))
    x = x.clip_lower(x.quantile(0.01))
    return x

custdata_df_num = custdata_df_num.apply(lambda x: outlier_capping(x))

Handling Missing Values¶

As we can see in the above audit report some data is missing in nmerical data set and some are missing in categorical.¶

We will treat numerical missing data with mean() and categorical data with mode()¶

#Handling missings - by Function (Makethe function to treat all data in one shot)

def Missing_imputation(x):
    x = x.fillna(x.mean())
    return x

custdata_df_num = custdata_df_num.apply(lambda x: Missing_imputation(x))

#Handling missings - by Function (Makethe function to treat all data in one shot)

def Missing_imputation(x):
    x = x.fillna(x.mode())
    return x

custdata_df_cat = custdata_df_cat.apply(lambda x: Missing_imputation(x))

Again check missing values has been treated or not¶

# Find the total number of missing values in the numerical data i.e custdata_df_num
print ("\nMissing values in numerical data :  ", custdata_df_num.isnull().sum().values.sum())
print ("\nMissing values in categorical data :  ", custdata_df_num.isnull().sum().values.sum())

Missing values in numerical data :   0

Missing values in categorical data :   0

Dummy creation for categorical data¶

# We need to create a function for dummy creation
def create_dummies(df, colname):
    col_dummies = pd.get_dummies(df[colname], prefix = colname)
    col_dummies.drop(col_dummies.columns[0], axis = 1, inplace = True)
    df = pd.concat([df, col_dummies], axis = 1)
    df.drop(colname, axis = 1, inplace = True)
    return df

custdata_df_cat=custdata_df.select_dtypes(include=['object'])
cat_varlist=list(custdata_df_cat.columns)

# For customer_features in categorical features

for c_feature in cat_varlist:
    custdata_df_cat[c_feature]=custdata_df_cat[c_feature].astype('category')
    custdata_df_cat=create_dummies(custdata_df_cat,c_feature)

custdata_df_cat.sample(5)

Merge Numerical and categorical data¶

Now we have nice simple clean data for numerical and categorical both data frame, so now we can merge both data set

custdata_df_new = pd.concat([custdata_df_num, custdata_df_cat], axis=1)

custdata_df_new.head()

# Some features has highly correlation with their log valuesso I am going to drop it.

#custdata_df_new.drop(['cardten', 'lninc','lncreddebt','lnothdebt','lnlongmon','lnlongten','lntollmon','lntollten',
#'lnequipmon','lnequipten','lncardten','lnwiremon','lnwireten'],axis=1, inplace=True)

custdata_df_new.shape

(5000, 400)

Explore data and check the variable distribution¶

# For Linear regressin y(totalspend) should follow normal istribution
import seaborn as sns
sns.distplot(custdata_df_new.totalspend)

<matplotlib.axes._subplots.AxesSubplot at 0xc5866a0>

Now as you can see distribution is right skewed so we need to take log and then plot the graph.

from scipy import stats
import pylab

stats.probplot(custdata_df_new.totalspend, dist="norm", plot=pylab )
pylab.show()

As we can see above graph our data is not distributed normally perfectly, so we will use boxcox techniue to make it perfect normal

from scipy import stats

# transform training data & save lambda value
custdata_df_new['bc_total_spend'],fitted_lambda = stats.boxcox(custdata_df_new['totalspend'])

sns.distplot(custdata_df_new.bc_total_spend)

<matplotlib.axes._subplots.AxesSubplot at 0xc010390>

stats.probplot(custdata_df_new.bc_total_spend, dist="norm", plot=pylab )
pylab.show()

#Drop the y variables(totalspend) from dataframe because we are looking correlation between all x variables

custdata_df_new.drop(['totalspend'],axis=1,inplace=True)

Devide data into train and test data¶

#Splitting the data in all x variable and y variable.

feature_columns=custdata_df_new.columns.difference(['bc_total_spend'])

from sklearn.model_selection import train_test_split

train_x,test_x,train_y,test_y=train_test_split(custdata_df_new[feature_columns],
                                              custdata_df_new['bc_total_spend'],
                                              test_size=0.2,
                                              random_state=12)

print (len(train_x))
print (len(test_x))
print (len(train_y))
print (len(test_y))

4000
1000
4000
1000

Feature selection by random forrest¶

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV

RandomForestRegressor?

param_grid={'n_estimators':np.arange(10,25)}

tree=GridSearchCV(RandomForestRegressor(oob_score=False,warm_start=True),param_grid,cv=2)
tree.fit(train_x,train_y)

GridSearchCV(cv=2, error_score='raise',
       estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
           oob_score=False, random_state=None, verbose=0, warm_start=True),
       fit_params=None, iid=True, n_jobs=1,
       param_grid={'n_estimators': array([10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24])},
       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',
       scoring=None, verbose=0)

tree.best_params_

{'n_estimators': 23}

# we can take n_estimators': 23
radm_clf = RandomForestRegressor(oob_score=True,n_estimators=23)
radm_clf.fit( train_x, train_y)

RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=23, n_jobs=1,
           oob_score=True, random_state=None, verbose=0, warm_start=False)

radm_clf.oob_score_

0.15767032576065787

from sklearn import metrics

indices = np.argsort(radm_clf.feature_importances_)[::-1]
feature_rank = pd.DataFrame( columns = ['rank', 'feature', 'importance'] )
for f in range(train_x.shape[1]):
  feature_rank.loc[f] = [f+1,
                         train_x.columns[indices[f]],
                         radm_clf.feature_importances_[indices[f]]]
sns.barplot( y = 'feature', x = 'importance', data = feature_rank )

<matplotlib.axes._subplots.AxesSubplot at 0xc09e4a8>

As above graph is not clear so we will use below method.

indices = np.argsort(radm_clf.feature_importances_)[::-1]
feature_rank = pd.DataFrame( columns = ['rank', 'feature', 'importance'] )
for f in range(train_x.shape[1]):
  feature_rank.loc[f] = [f+1,
                         train_x.columns[indices[f]],
                         radm_clf.feature_importances_[indices[f]]]

feature_rank

#Select features and then convert it into list
x=feature_rank.loc[0:75,['feature']]
x=x['feature'].tolist()
print(x)

['income', 'lninc', 'card_3', 'card_2', 'carvalue', 'card_4', 'reason_2', 'debtinc', 'commutetime', 'hourstv', 'age', 'lncreddebt', 'creddebt', 'othdebt', 'ed', 'lncardmon', 'tenure', 'lnothdebt', 'pets', 'lncardten', 'cardmon', 'lntollmon', 'longmon', 'card_5', 'spoused', 'cardten', 'lnlongten', 'lnequipten', 'lnlongmon', 'longten', 'card2_2', 'lntollten', 'lnequipmon', 'tollmon', 'card2_3', 'lnwiremon', 'gender_1', 'reside', 'tollten', 'pets_cats', 'pets_dogs', 'pets_freshfish', 'equipmon', 'commutewalk_1', 'polparty_1', 'lnwireten', 'polview_4', 'card2benefit_3', 'union_1', 'spousedcat_2', 'card2benefit_2', 'card2type_2', 'vote_1', 'carown_0', 'equipten', 'region_3', 'card2_4', 'carbought_1', 'carbuy_1', 'jobsat_4', 'birthmonth_October', 'active_1', 'townsize_3.0', 'cardbenefit_4', 'region_5', 'cartype_0', 'cardtype_2', 'cardtype_4', 'forward_1', 'commutebus_1', 'jobsat_3', 'commutecar_1', 'card2benefit_4', 'townsize_4.0', 'commuterail_1', 'wiremon']

# Create data frame with selected features
rf_features=['income', 'lninc', 'card_3', 'card_2', 'carvalue', 'card_4', 'commutetime', 'reason_2', 'debtinc', 'hourstv', 'creddebt', 'age', 'lncreddebt', 'tenure', 'lncardmon', 'lnothdebt', 'ed', 'othdebt', 'pets', 'lncardten', 'lntollmon', 'cardmon', 'longmon', 'card_5', 'lnlongmon', 'reside', 'spoused', 'cardten', 'lntollten', 'lnequipmon', 'lnlongten', 'tollten', 'longten', 'tollmon', 'lnwiremon', 'card2_2', 'card2_3', 'pets_freshfish', 'lnequipten', 'gender_1', 'pets_dogs', 'equipten', 'wireten', 'pets_cats', 'address_22', 'equipmon', 'wiremon', 'jobsat_4', 'commutebus_1', 'card2benefit_4']

rf_features.append('bc_total_spend')
df_rf= custdata_df_new[rf_features]

df_rf.head(5)

Check - 2: Find multicolinearlity¶

import statsmodels as sm
from statsmodels.stats.outliers_influence import variance_inflation_factor
from patsy import dmatrices

del rf_features[-1]

%%capture
#gather features
features = "+".join(rf_features)

features

'income+lninc+card_3+card_2+carvalue+card_4+commutetime+reason_2+debtinc+hourstv+creddebt+age+lncreddebt+tenure+lncardmon+lnothdebt+ed+othdebt+pets+lncardten+lntollmon+cardmon+longmon+card_5+lnlongmon+reside+spoused+cardten+lntollten+lnequipmon+lnlongten+tollten+longten+tollmon+lnwiremon+card2_2+card2_3+pets_freshfish+lnequipten+gender_1+pets_dogs+equipten+wireten+pets_cats+address_22+equipmon+wiremon+jobsat_4+commutebus_1+card2benefit_4'

# get y and X dataframes based on this regression:
y, X = dmatrices('bc_total_spend~' + features,df_rf, return_type='dataframe')

# For each X, calculate VIF and save in dataframe
vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
vif["features"] = X.columns

vif.sort_values(by=['VIF Factor'],ascending=False)

#Select only those features whose Vif is greater than 10.
vif1=vif[vif['VIF Factor']>10].reset_index().loc[:,['features']]
vif1.drop([0],axis=0,inplace=True)
drop_vars= vif1["features"].tolist()
drop_vars

['tenure',
 'pets',
 'cardmon',
 'longmon',
 'lnlongmon',
 'cardten',
 'lnlongten',
 'tollten',
 'longten',
 'tollmon',
 'pets_freshfish',
 'equipten',
 'wireten',
 'equipmon',
 'wiremon']

# dropping variables that have VIF greater than 10
df_rf.drop(drop_vars,axis=1,inplace=True)

df_rf.shape

(5000, 36)

Check 3 - All x variables should have a linear relationship with Y¶

sns.lmplot(x="income",y="bc_total_spend",data=df_rf,aspect=5,scatter_kws={'alpha':0.5})

<seaborn.axisgrid.FacetGrid at 0xb319320>

Splitting data for training and testing¶

# Dropping variables one at a time which have p-values greater than 5%
feature_columns=df_rf.columns.difference(['bc_total_spend','address_22','pets_cats','lncardmon','lnequipten','spoused'
                                         ,'creddebt','reside','othdebt','debtinc','lnothdebt','jobsat_4','income','carvalue',
                                         'card2benefit_4','commutebus_1','hourstv','lnwiremon','pets_dogs','lncardten',
                                         'commutetime','lntollten','lntollmon','lnequipmon'])

train_x,test_x,train_y,test_y=train_test_split(df_rf[feature_columns],
                                              df_rf['bc_total_spend'],
                                               test_size=.2,
                                               random_state=123)

Build Regression model using statsmodels.api¶

import statsmodels.api as sm

train_x = sm.add_constant(train_x)
test_x=sm.add_constant(test_x)
lm=sm.OLS(train_y,train_x).fit()

print(lm.summary())

                            OLS Regression Results                            
==============================================================================
Dep. Variable:         bc_total_spend   R-squared:                       0.325
Model:                            OLS   Adj. R-squared:                  0.323
Method:                 Least Squares   F-statistic:                     159.7
Date:                Tue, 01 Oct 2019   Prob (F-statistic):               0.00
Time:                        22:46:22   Log-Likelihood:                -6598.7
No. Observations:                4000   AIC:                         1.322e+04
Df Residuals:                    3987   BIC:                         1.331e+04
Df Model:                          12                                         
Covariance Type:            nonrobust                                         
==============================================================================
                 coef    std err          t      P>|t|      [0.025      0.975]
------------------------------------------------------------------------------
const          8.2401      0.172     47.920      0.000       7.903       8.577
age           -0.0042      0.001     -3.681      0.000      -0.006      -0.002
card2_2       -0.3794      0.052     -7.349      0.000      -0.481      -0.278
card2_3       -0.3452      0.051     -6.788      0.000      -0.445      -0.246
card_2        -1.2433      0.064    -19.525      0.000      -1.368      -1.118
card_3        -1.2391      0.064    -19.465      0.000      -1.364      -1.114
card_4        -1.2693      0.060    -21.022      0.000      -1.388      -1.151
card_5        -0.9928      0.108     -9.214      0.000      -1.204      -0.782
ed            -0.0154      0.007     -2.349      0.019      -0.028      -0.003
gender_1      -0.1489      0.040     -3.713      0.000      -0.227      -0.070
lncreddebt     0.0396      0.020      1.997      0.046       0.001       0.079
lninc          0.8090      0.034     23.600      0.000       0.742       0.876
reason_2       0.7071      0.081      8.746      0.000       0.549       0.866
==============================================================================
Omnibus:                        4.553   Durbin-Watson:                   1.976
Prob(Omnibus):                  0.103   Jarque-Bera (JB):                4.573
Skew:                          -0.082   Prob(JB):                        0.102
Kurtosis:                       2.983   Cond. No.                         473.
==============================================================================

Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

lm.pvalues.sort_values(ascending=False)

lncreddebt     4.587652e-02
ed             1.887734e-02
age            2.357317e-04
gender_1       2.078595e-04
card2_3        1.303811e-11
card2_2        2.408523e-13
reason_2       3.210677e-18
card_5         4.947423e-20
card_3         1.090371e-80
card_2         3.707595e-81
card_4         3.890897e-93
lninc         2.235272e-115
const          0.000000e+00
dtype: float64

print('Parameters:', lm.params) # Find the parameters of x i.e Beta value
print('R2: ', lm.rsquared) # Find the r**2

Parameters: const         8.240115
age          -0.004194
card2_2      -0.379415
card2_3      -0.345223
card_2       -1.243274
card_3       -1.239082
card_4       -1.269314
card_5       -0.992801
ed           -0.015444
gender_1     -0.148872
lncreddebt    0.039624
lninc         0.809044
reason_2      0.707096
dtype: float64
R2:  0.3246950341215388

Eveluation of model accuracy¶

test_pred=lm.predict(test_x)
train_pred=lm.predict(train_x)

from sklearn import metrics

print('MSE Test:',metrics.mean_squared_error(test_y,test_pred))
print('MSE Train:',metrics.mean_squared_error(train_y,train_pred))

MSE Test: 1.5275403610908898
MSE Train: 1.5864409028377782

print ('MAE:', metrics.mean_absolute_error(test_y, test_pred))
print ('MSE:', metrics.mean_squared_error(test_y, test_pred))
print ('RMSE:', np.sqrt(metrics.mean_squared_error(test_y, test_pred)))

MAE: 0.9762061585444958
MSE: 1.5275403610908898
RMSE: 1.2359370376725871

MAPE_train = '%.3f' % np.mean(np.abs(train_y-train_pred)/(train_y))
MAPE_test = '%.3f' % np.mean(np.abs(test_y-test_pred)/(test_y))

# print the values of MAPE for train and test
print('MAPE of training data: ', MAPE_train,  ' | ', 'MAPE of testing data: ', MAPE_test)

MAPE of training data:  0.110  |  MAPE of testing data:  0.106

Check Normality and Residuals¶

We will use Q-Q plot to examine this

residuals=train_y-train_pred

import seaborn as sns

sns.distplot(residuals)

<matplotlib.axes._subplots.AxesSubplot at 0xf889278>

from scipy import stats
import pylab

stats.probplot(residuals,dist='norm',plot=pylab)
pylab.show()

From the above histogram and Q-Q plot, shows that the residuals are normally distributed, so our assumption is not violated

Predicting sales:¶

We are going to credit card sales on Testing data

df_rf=df_rf[feature_columns]

df_rf=sm.add_constant(df_rf)

df_rf.head()

# Final prediction on test file
from scipy.special import boxcox, inv_boxcox

pred_total_spend=inv_boxcox(lm.predict(df_rf),fitted_lambda)
pred_total_spend=pd.DataFrame(pred_total_spend.tolist())
pred_total_spend.columns = ['pred_total_spend']


#Concatenating final prediction with original test file
  
testfile = pd.concat([custdata_df, pred_total_spend], axis=1)

testfile.head()

#Export the final result in csv.
%cd C:\\Users\\ashwini\\Desktop
testfile.to_csv('Final_submission_credit_spend.csv',index=False)

C:\Users\ashwini\Desktop

##################End######################

Number of variables	131
Number of observations	5000
Total Missing (%)	0.2%
Total size in memory	5.0 MiB
Average record size in memory	1.0 KiB

Distinct count	2
Unique (%)	0.0%
Missing (%)	0.0%
Missing (n)	0

Value	Count	Frequency (%)
0	2670	53.4%
1	2330	46.6%

Distinct count	57
Unique (%)	1.1%
Missing (%)	0.0%
Missing (n)	0
Infinite (%)	0.0%
Infinite (n)	0

Mean	16.402
Minimum	0
Maximum	57
Zeros (%)	4.9%